#coding=utf-8
from masterRedis import masterRedis
from brokerRedis import brokerRedis
import os
import ConfigParser
#from pybloomfilter import BloomFilter
class master:
    def __init__(self,url):
        self.brokerRedis=brokerRedis()
        self.masterRedis=masterRedis()
        self.config=ConfigParser.ConfigParser()
        path = os.path.split(os.path.realpath(__file__))[0] + '/config.conf'
        self.config.read(path)
        path="bf_file.txt"
        if os.path.exists(path):
            print "open path"
            self.bf=BloomFilter.open(path)
        else:
            self.bf=BloomFilter(20000000,0.01,path)
        self.addUrl(url)
        self.getUrl()
    
    def check(self,url):
        if url not in self.bf:
            self.bf.add(url)
            return True
        return False
        
    def addUrl(self,link):
        if self.check(link):
            self.masterRedis.saveUrl(link)
    
    def getUrl(self):
        while True:
            try:
                self.addUrl(self.config.get("urlHeaders", "url")+self.brokerRedis.getUrl()[1])
            except Exception as e:
                print e
            
if __name__ == "__main__" :
    ma=master("http://baike.baidu.com/link?url=sag3Vj9DEB1LmvlMfbGJlwPD3fzLmcj9W7PevohyvntkbaZAnk0L-Yfq_0bE6PpEtYKaTaTX9VqJzbX0bCjFONTLFBsYXRY0trwZXuiD3kM9LnRJLVAU_7TqV3Lb4aT-")